<!DOCTYPE html>
<html lang="en-us">
  <head>

    <meta http-equiv="content-type" content="text/html; charset=utf-8">
    
<meta charset="UTF-8">
<title>Making Changes Persistent | Elasticsearch: The Definitive Guide [2.x] | Elastic</title>
<link rel="home" href="index.html" title="Elasticsearch: The Definitive Guide [2.x]">
<link rel="up" href="inside-a-shard.html" title="Inside a Shard">
<link rel="prev" href="near-real-time.html" title="Near Real-Time Search">
<link rel="next" href="merge-process.html" title="Segment Merging">
<meta name="DC.type" content="Learn/Docs/Legacy/Elasticsearch/Definitive Guide/2.x">
<meta name="DC.subject" content="Elasticsearch">
<meta name="DC.identifier" content="2.x">
<meta name="robots" content="noindex,nofollow">
    <meta http-equiv="X-UA-Compatible" content="IE=edge">
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <script src="https://cdn.optimizely.com/js/18132920325.js"></script>
    <link rel="apple-touch-icon" sizes="57x57" href="/apple-icon-57x57.png">
    <link rel="apple-touch-icon" sizes="60x60" href="/apple-icon-60x60.png">
    <link rel="apple-touch-icon" sizes="72x72" href="/apple-icon-72x72.png">
    <link rel="apple-touch-icon" sizes="76x76" href="/apple-icon-76x76.png">
    <link rel="apple-touch-icon" sizes="114x114" href="/apple-icon-114x114.png">
    <link rel="apple-touch-icon" sizes="120x120" href="/apple-icon-120x120.png">
    <link rel="apple-touch-icon" sizes="144x144" href="/apple-icon-144x144.png">
    <link rel="apple-touch-icon" sizes="152x152" href="/apple-icon-152x152.png">
    <link rel="apple-touch-icon" sizes="180x180" href="/apple-icon-180x180.png">
    <link rel="icon" type="image/png" href="/favicon-32x32.png" sizes="32x32">
    <link rel="icon" type="image/png" href="/android-chrome-192x192.png" sizes="192x192">
    <link rel="icon" type="image/png" href="/favicon-96x96.png" sizes="96x96">
    <link rel="icon" type="image/png" href="/favicon-16x16.png" sizes="16x16">
    <link rel="manifest" href="/manifest.json">
    <meta name="apple-mobile-web-app-title" content="Elastic">
    <meta name="application-name" content="Elastic">
    <meta name="msapplication-TileColor" content="#ffffff">
    <meta name="msapplication-TileImage" content="/mstile-144x144.png">
    <meta name="theme-color" content="#ffffff">
    <meta name="naver-site-verification" content="936882c1853b701b3cef3721758d80535413dbfd">
    <meta name="yandex-verification" content="d8a47e95d0972434">
    <meta name="localized" content="true">
    <meta name="st:robots" content="follow,index">
    <meta property="og:image" content="https://www.elastic.co/static/images/elastic-logo-200.png">
    <link rel="shortcut icon" href="/favicon.ico" type="image/x-icon">
    <link rel="icon" href="/favicon.ico" type="image/x-icon">
    <link rel="apple-touch-icon-precomposed" sizes="64x64" href="/favicon_64x64_16bit.png">
    <link rel="apple-touch-icon-precomposed" sizes="32x32" href="/favicon_32x32.png">
    <link rel="apple-touch-icon-precomposed" sizes="16x16" href="/favicon_16x16.png">
    <!-- Give IE8 a fighting chance -->
    <!--[if lt IE 9]>
    <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
    <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
    <![endif]-->
    <link rel="stylesheet" type="text/css" href="/guide/static/styles.css">
  </head>

  <!--© 2015-2021 Elasticsearch B.V. Copying, publishing and/or distributing without written permission is strictly prohibited.-->

  <body>
    <!-- Google Tag Manager -->
    <script>dataLayer = [];</script><noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-58RLH5" height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start': new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0], j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src= '//www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f); })(window,document,'script','dataLayer','GTM-58RLH5');</script>
    <!-- End Google Tag Manager -->

    <!-- Global site tag (gtag.js) - Google Analytics -->
    <script async src="https://www.googletagmanager.com/gtag/js?id=UA-12395217-16"></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag(){dataLayer.push(arguments);}
      gtag('js', new Date());
      gtag('config', 'UA-12395217-16');
    </script>

    <!--BEGIN QUALTRICS WEBSITE FEEDBACK SNIPPET-->
    <script type="text/javascript">
      (function(){var g=function(e,h,f,g){
      this.get=function(a){for(var a=a+"=",c=document.cookie.split(";"),b=0,e=c.length;b<e;b++){for(var d=c[b];" "==d.charAt(0);)d=d.substring(1,d.length);if(0==d.indexOf(a))return d.substring(a.length,d.length)}return null};
      this.set=function(a,c){var b="",b=new Date;b.setTime(b.getTime()+6048E5);b="; expires="+b.toGMTString();document.cookie=a+"="+c+b+"; path=/; "};
      this.check=function(){var a=this.get(f);if(a)a=a.split(":");else if(100!=e)"v"==h&&(e=Math.random()>=e/100?0:100),a=[h,e,0],this.set(f,a.join(":"));else return!0;var c=a[1];if(100==c)return!0;switch(a[0]){case "v":return!1;case "r":return c=a[2]%Math.floor(100/c),a[2]++,this.set(f,a.join(":")),!c}return!0};
      this.go=function(){if(this.check()){var a=document.createElement("script");a.type="text/javascript";a.src=g;document.body&&document.body.appendChild(a)}};
      this.start=function(){var a=this;window.addEventListener?window.addEventListener("load",function(){a.go()},!1):window.attachEvent&&window.attachEvent("onload",function(){a.go()})}};
      try{(new g(100,"r","QSI_S_ZN_emkP0oSe9Qrn7kF","https://znemkp0ose9qrn7kf-elastic.siteintercept.qualtrics.com/WRSiteInterceptEngine/?Q_ZID=ZN_emkP0oSe9Qrn7kF")).start()}catch(i){}})();
    </script><div id="ZN_emkP0oSe9Qrn7kF"><!--DO NOT REMOVE-CONTENTS PLACED HERE--></div>
    <!--END WEBSITE FEEDBACK SNIPPET-->

    <div id="elastic-nav" style="display:none;"></div>
    <script src="https://www.elastic.co/elastic-nav.js"></script>

    <!-- Subnav -->
    <div>
      <div>
        <div class="tertiary-nav d-none d-md-block">
          <div class="container">
            <div class="p-t-b-15 d-flex justify-content-between nav-container">
              <div class="breadcrum-wrapper"><span><a href="/guide/" style="font-size: 14px; font-weight: 600; color: #000;">Docs</a></span></div>
            </div>
          </div>
        </div>
      </div>
    </div>

    <div class="main-container">
      <section id="content">
        <div class="content-wrapper">

          <section id="guide" lang="en">
            <div class="container">
              <div class="row">
                <div class="col-xs-12 col-sm-8 col-md-8 guide-section">
                  <!-- start body -->
                  <div class="page_header">
<p>
  <strong>WARNING</strong>: The 2.x versions of Elasticsearch have passed their
  <a href="https://www.elastic.co/support/eol">EOL dates</a>. If you are running
  a 2.x version, we strongly advise you to upgrade.
</p>
<p>
  This documentation is no longer maintained and may be removed. For the latest
  information, see the <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html">current
  Elasticsearch documentation</a>.
</p>
</div>
<div id="content">
<div class="breadcrumbs">
<span class="breadcrumb-link"><a href="index.html">Elasticsearch: The Definitive Guide [2.x]</a></span>
»
<span class="breadcrumb-link"><a href="getting-started.html">Getting Started</a></span>
»
<span class="breadcrumb-link"><a href="inside-a-shard.html">Inside a Shard</a></span>
»
<span class="breadcrumb-node">Making Changes Persistent</span>
</div>
<div class="navheader">
<span class="prev">
<a href="near-real-time.html">« Near Real-Time Search</a>
</span>
<span class="next">
<a href="merge-process.html">Segment Merging »</a>
</span>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h2 class="title">
<a id="translog"></a>Making Changes Persistent<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/075_Inside_a_shard/50_Persistent_changes.asciidoc">edit</a>
</h2>
</div></div></div>
<p>Without an <code class="literal">fsync</code> to flush data in the filesystem cache to disk, we cannot
be sure that the data will still be there after a power failure, or even after
exiting the application normally.  For Elasticsearch to be reliable, it needs
to ensure that changes are persisted to disk.</p>
<p>In <a class="xref" href="dynamic-indices.html" title="Dynamically Updatable Indices">Dynamically Updatable Indices</a>, we said that a full commit flushes segments to disk and
writes a commit point, which lists all known segments.  Elasticsearch uses
this commit point during startup or when reopening an index to decide which
segments belong to the current shard.</p>
<p>While we refresh once every second to achieve near real-time search, we still
need to do full commits regularly to make sure that we can recover from
failure.  But what about the document changes that happen between commits?  We
don’t want to lose those either.</p>
<p>Elasticsearch added a <em>translog</em>, or transaction log, which records every
operation in Elasticsearch as it happens.  With the translog, the process now
looks like this:</p>
<div class="olist orderedlist">
<ol class="orderedlist">
<li class="listitem">
<p>When a document is indexed, it is added to the in-memory buffer <em>and</em>
appended to the translog, as shown in <a class="xref" href="translog.html#img-xlog-pre-refresh" title="New documents are added to the in-memory buffer and appended to the transaction log">Figure 21, “New documents are added to the in-memory buffer and appended to the transaction log”</a>.</p>
<div id="img-xlog-pre-refresh" class="imageblock">
<div class="content">
<img src="images/elas_1106.png" alt="New documents are added to the in-memory buffer and appended to the transaction log">
</div>
<div class="title">Figure 21. New documents are added to the in-memory buffer and appended to the transaction log</div>
</div>
</li>
<li class="listitem">
<p>The refresh leaves the shard in the state depicted in <a class="xref" href="translog.html#img-xlog-post-refresh" title="After a refresh, the buffer is cleared but the transaction log is not">Figure 22, “After a refresh, the buffer is cleared but the transaction log is not”</a>. Once every second, the shard is refreshed:</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
The docs in the in-memory buffer are written to a new segment,
without an <code class="literal">fsync</code>.
</li>
<li class="listitem">
The segment is opened to make it visible to search.
</li>
<li class="listitem">
The in-memory buffer is cleared.
</li>
</ul>
</div>
<div id="img-xlog-post-refresh" class="imageblock">
<div class="content">
<img src="images/elas_1107.png" alt="After a refresh, the buffer is cleared but the transaction log is not">
</div>
<div class="title">Figure 22. After a refresh, the buffer is cleared but the transaction log is not</div>
</div>
</li>
<li class="listitem">
<p>This process continues with more documents being added to the in-memory
buffer and appended to the transaction log (see <a class="xref" href="translog.html#img-xlog-pre-flush" title="The transaction log keeps accumulating documents">Figure 23, “The transaction log keeps accumulating documents”</a>).</p>
<div id="img-xlog-pre-flush" class="imageblock">
<div class="content">
<img src="images/elas_1108.png" alt="The transaction log keeps accumulating documents">
</div>
<div class="title">Figure 23. The transaction log keeps accumulating documents</div>
</div>
</li>
<li class="listitem">
<p>Every so often—​such as when the translog is getting too big—​the index
is flushed; a new translog is created, and a full commit is performed (see <a class="xref" href="translog.html#img-xlog-post-flush" title="After a flush, the segments are fully commited and the transaction log is cleared">Figure 24, “After a flush, the segments are fully commited and the transaction log is cleared”</a>):</p>
<div class="ulist itemizedlist">
<ul class="itemizedlist">
<li class="listitem">
Any docs in the in-memory buffer are written to a new segment.
</li>
<li class="listitem">
The buffer is cleared.
</li>
<li class="listitem">
A commit point is written to disk.
</li>
<li class="listitem">
The filesystem cache is flushed with an <code class="literal">fsync</code>.
</li>
<li class="listitem">
The old translog is deleted.
</li>
</ul>
</div>
</li>
</ol>
</div>
<p>The translog provides a persistent record of all operations that have not yet
been flushed to disk. When starting up, Elasticsearch will use the last commit
point to recover known segments from disk, and will then replay all operations
in the translog to add the changes that happened after the last commit.</p>
<p>The translog is also used to provide real-time CRUD.  When you try to
retrieve, update, or delete a document by ID, it first checks the translog for
any recent changes before trying to retrieve the document from the relevant
segment. This means that it always has access to the latest known version of
the document, in real-time.</p>
<div id="img-xlog-post-flush" class="imageblock">
<div class="content">
<img src="images/elas_1109.png" alt="After a flush, the segments are fully commited and the transaction log is cleared">
</div>
<div class="title">Figure 24. After a flush, the segments are fully commited and the transaction log is cleared</div>
</div>
<div class="section">
<div class="titlepage"><div><div>
<h3 class="title">
<a id="flush-api"></a>flush API<a class="edit_me edit_me_private" rel="nofollow" title="Editing on GitHub is available to Elastic" href="https://github.com/elastic/elasticsearch-definitive-guide/edit/2.x/075_Inside_a_shard/50_Persistent_changes.asciidoc">edit</a>
</h3>
</div></div></div>
<p>The action of performing a commit and truncating the translog is known in
Elasticsearch as a <em>flush</em>.  Shards are flushed automatically every 30
minutes, or when the translog becomes too big. See the
<a href="/guide/en/elasticsearch/reference/2.4/index-modules-translog.html#_translog_settings" class="ulink" target="_top"><code class="literal">translog</code> documentation</a> for settings
that can be used to control these thresholds:</p>
<p>The <a href="/guide/en/elasticsearch/reference/2.4/indices-flush.html" class="ulink" target="_top"><code class="literal">flush</code> API</a> can be used to perform a manual flush:</p>
<div class="pre_wrapper lang-json">
<pre class="programlisting prettyprint lang-json">POST /blogs/_flush <a id="CO40-1"></a><i class="conum" data-value="1"></i>

POST /_flush?wait_for_ongoing <a id="CO40-2"></a><i class="conum" data-value="2"></i></pre>
</div>
<div class="calloutlist">
<table border="0" summary="Callout list">
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO40-1"><i class="conum" data-value="1"></i></a></p>
</td>
<td align="left" valign="top">
<p>Flush the <code class="literal">blogs</code> index.</p>
</td>
</tr>
<tr>
<td align="left" valign="top" width="5%">
<p><a href="#CO40-2"><i class="conum" data-value="2"></i></a></p>
</td>
<td align="left" valign="top">
<p>Flush all indices and wait until all flushes have completed before
returning.</p>
</td>
</tr>
</table>
</div>
<p>You seldom need to issue a manual <code class="literal">flush</code> yourself; usually, automatic
flushing is all that is required.</p>
<p>That said, it is beneficial to <a class="xref" href="translog.html#flush-api" title="flush API">flush</a> your indices before restarting a node or closing an index. When Elasticsearch tries to recover or reopen an index, it has to replay all of the operations in the translog, so the shorter the log, the faster the recovery.</p>
<div class="sidebar">
<a id="how-safe-is-the-translog"></a>
<div class="titlepage"><div><div>
<p class="title"><strong>How Safe Is the Translog?</strong></p>
</div></div></div>
<p>The purpose of the translog is to ensure that operations are not lost.  This
begs the question: how safe is the translog?</p>
<p>Writes to a file will not survive a reboot until the file has been
<code class="literal">fsync</code>'ed to disk.  By default, the translog is <code class="literal">fsync</code>'ed every 5
seconds <em>and</em> after a write request completes (e.g. index, delete, update, bulk).
This process occurs on both the primary and replica shards. Ultimately, that means
your client won’t receive a <code class="literal">200 OK</code> response until the entire request has been
<code class="literal">fsync</code>'ed in the translog of the primary and all replicas.</p>
<p>Executing an fsync after every request does come with some performance cost,
although in practice it is relatively small (especially for bulk ingestion, which
amortizes the cost over many documents in the single request).</p>
<p>But for some high-volume clusters where losing a few seconds of data is not
critical, it can be advantageous to fsync <em>asynchronously</em>.  E.g. writes are
buffered in memory and <code class="literal">fsync</code>'ed together every 5s.</p>
<p>This behavior can be enabled by setting the <code class="literal">durability</code> parameter to <code class="literal">async</code>:</p>
<div class="pre_wrapper lang-js">
<pre class="programlisting prettyprint lang-js">PUT /my_index/_settings
{
    "index.translog.durability": "async",
    "index.translog.sync_interval": "5s"
}</pre>
</div>
<p>This setting can be configured per-index and is dynamically updatable. If
you decide to enable async translog behavior, you are <em>guaranteed</em> to lose
<code class="literal">sync_interval</code>'s worth of data if a crash happens.  Please be aware of this
characteristic before deciding!</p>
<p>If you are unsure the ramifications of this action, it is best to use the default
(<code class="literal">"index.translog.durability": "request"</code>) to avoid data-loss.</p>
</div>
</div>

</div>
<div class="navfooter">
<span class="prev">
<a href="near-real-time.html">« Near Real-Time Search</a>
</span>
<span class="next">
<a href="merge-process.html">Segment Merging »</a>
</span>
</div>
</div>

                  <!-- end body -->
                </div>
                <div class="col-xs-12 col-sm-4 col-md-4" id="right_col">
                  <div id="rtpcontainer" style="display: block;">
                    <div class="mktg-promo">
                      <h3>Most Popular</h3>
                      <ul class="icons">
                        <li class="icon-elasticsearch-white"><a href="https://www.elastic.co/webinars/getting-started-elasticsearch?baymax=default&amp;elektra=docs&amp;storm=top-video">Get Started with Elasticsearch: Video</a></li>
                        <li class="icon-kibana-white"><a href="https://www.elastic.co/webinars/getting-started-kibana?baymax=default&amp;elektra=docs&amp;storm=top-video">Intro to Kibana: Video</a></li>
                        <li class="icon-logstash-white"><a href="https://www.elastic.co/webinars/introduction-elk-stack?baymax=default&amp;elektra=docs&amp;storm=top-video">ELK for Logs &amp; Metrics: Video</a></li>
                      </ul>
                    </div>
                  </div>
                </div>
              </div>
            </div>
          </section>

        </div>


<div id="elastic-footer"></div>
<script src="https://www.elastic.co/elastic-footer.js"></script>
<!-- Footer Section end-->

      </section>
    </div>

<script src="/guide/static/jquery.js"></script>
<script type="text/javascript" src="/guide/static/docs.js"></script>
<script type="text/javascript">
  window.initial_state = {}</script>
  </body>
</html>
